package com.tzxx.spider.spiderbots;

import lombok.extern.slf4j.Slf4j;
import us.codecraft.webmagic.Page;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import com.tzxx.spider.domain.ArticleLink;
import com.tzxx.spider.domain.Template;
import com.tzxx.spider.service.ArticleLinkService;
import com.tzxx.spider.spiderbots.base.BaseUrlListSpider;
import com.tzxx.spider.utils.HtmlUtils;
import com.tzxx.spider.utils.SpringContextUtil;

/**
 * @author Created by tzxx
 * @since : 2019/7/18.
 */
@Slf4j
public class ListSpider extends BaseUrlListSpider {

    private int startPageNum = 1;

    public ListSpider(Template template) {
        super(template);
    }

    @Override
    public void process(Page page) {
        ArticleLinkService articleLinkService = SpringContextUtil.getBean("articleLinkService");
        // 文章url列表
        List<String> articlesUrl = page.getHtml().xpath(template.getListItemSelector()).all();
        List<String> newUrl = new ArrayList<>();
        Iterator<String> articlesUrlIterator = articlesUrl.iterator();
        while (articlesUrlIterator.hasNext()) {
            String url = articlesUrlIterator.next();
            //处理不规范url链接
            if (!url.startsWith("http")) {
                newUrl.add(HtmlUtils.handleUrl(page.getUrl().get(), url));
                articlesUrlIterator.remove();
            }
        }
        articlesUrl.addAll(newUrl);
        // 若拿不到链接则说明已经超出了最后一页
        if (newUrl.isEmpty()) {
            page.setSkip(true);
            return;
        }
        //链接
        List<ArticleLink> articleLinks = new ArrayList<>();
        ArticleLink articleLink;
        for (String url : newUrl) {
            articleLink = new ArticleLink();
            articleLink.setTemplateId(template.getId());
            articleLink.setUrl(url);
            articleLinks.add(articleLink);
        }
        //存入redis队列，交给内容爬虫去爬取详情
        articleLinkService.insertList2Redis(articleLinks);
        //存入mysql
//        articleLinkService.insertList2Mysql(articleLinks);
        articleLinks.clear();
        //下一页url逻辑
        this.nextPageUrl(page);
    }

    /**
     * //下一页url逻辑
     *
     * @param page
     */
    @Override
    protected void nextPageUrl(Page page) {
        if (startPageNum <= template.getListUrlMaxNum()) {
            String nextUrl = template.getListUrlRule().replace("#{next}#", startPageNum + "");
            page.addTargetRequest(nextUrl);
            startPageNum++;
        }
    }

}
